package com.starry.crawler_shop.webmagic;

import com.alibaba.fastjson.JSONObject;
import com.starry.crawler_shop.entity.Product;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.jsoup.nodes.Document;
import org.springframework.util.CollectionUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

public class GMProcessor implements PageProcessor
{

	private static PoolingHttpClientConnectionManager poolManger;
	private static RequestConfig build;

	public static CloseableHttpClient getHttpClient() {
		if (poolManger == null) {

			//配置连接池的参数
			poolManger = new PoolingHttpClientConnectionManager();

			//最大连接数
			poolManger.setMaxTotal(100);
			//路由最大连接数
			poolManger.setDefaultMaxPerRoute(2);
		}
		if (build == null) {

			//配置连接池中连接的参数
			build = RequestConfig.custom().setConnectTimeout(5000)//发送请求的超时时间
					.setSocketTimeout(2000)//响应超时时间
					.setConnectionRequestTimeout(500)//从连接池中获取的超时时间
//					.setProxy(new HttpHost("39.106.223.134", 80))
					.build();

		}

		//拿到httpClient
		CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(poolManger)
				.setDefaultRequestConfig(build)
				.build();



		return httpClient;

	}

	@Override
	public void process(Page page) {

		//每次爬取的间隔时间 1-3秒
		this.site.setSleepTime((int) (Math.random() * 2000 + 1000)); //间隔时间


		if (page.getHtml().css("#product-box").get() != null) {

			List<String> detailLinks = page.getHtml().css("#product-box li .item-pic .item-link").links().all();
			System.out.println(detailLinks);
//			page.addTargetRequests(detailLinks);
			System.out.println(detailLinks.size());

		} else {
			//详情页
			Document dom = page.getHtml().getDocument();
			//商品标题
			String title = dom.select("#itemDisplayName").text();
			String skuImg = dom.select("#bigImg img").attr("src");
			//商品地址
			String url = page.getUrl().get();
			Product product = new Product();
			String codes = url.substring(url.indexOf("product.suning.com/"), url.lastIndexOf(".html"));
			String[] codesArray = codes.split("/");
			String bizCode = codesArray[1];
			String commCode = codesArray[2];
			String priceUrl = "https://pas.suning.com/nspcsale_0_0000000" + commCode + "_0000000" + commCode + "_" + bizCode + "_10_010_0100100_20089_1000000_9017_10106_Z001___R1901001_0.97_3___000060021_01___0___1.0_2_.html";
			HttpGet httpGet = new HttpGet(priceUrl);
			httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36");
			try {
				CloseableHttpResponse response = getHttpClient().execute(httpGet);
				if (response.getStatusLine().getStatusCode() == 200) {
					//请求成功
					HttpEntity entity = response.getEntity();
					String json = EntityUtils.toString(entity, "UTF-8");
					if (StringUtils.isNotBlank(json)) {
						String substring = json.substring(json.indexOf("(") + 1, json.lastIndexOf(")"));
						Map<String, Object> parse = (Map<String, Object>) JSONObject.parse(substring);
						Map<String, Object> data = (Map<String, Object>) parse.get("data");
						Map<String, Object> price = (Map<String, Object>) data.get("price");
						List<Map<String, Object>> saleInfo = (List<Map<String, Object>>) price.get("saleInfo");
						Map<String, Object> map = saleInfo.get(0);
						String promotionPrice = (String) map.get("promotionPrice");
						double v = Double.parseDouble(promotionPrice);
						product.setPrice(v);

					}

				}


			} catch (IOException e) {
				e.printStackTrace();
			}

			String clusterId = "1111";
			String html = dom.html();
			String str = html.substring(html.indexOf("clusterId") + 12);
			clusterId = str.substring(0, str.indexOf("\""));
			if (StringUtils.isBlank(clusterId)) {
				clusterId = "1234";
			}
			HttpGet total = new HttpGet("https://review.suning.com/ajax/review_count/cluster-" + clusterId + "-0000000" + commCode + "-" + bizCode + "-----satisfy.htm");
			total.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36");

			try {
				CloseableHttpResponse response = getHttpClient().execute(total);
				if (response.getStatusLine().getStatusCode() == 200) {
					String json = EntityUtils.toString(response.getEntity(), "UTF-8");
					String substring = json.substring(json.indexOf("(") + 1, json.lastIndexOf(")"));
					Map<String, Object> parse = (Map<String, Object>) JSONObject.parse(substring);
					List<Map<String, Object>> reviewCounts = (List<Map<String, Object>>) parse.get("reviewCounts");
					if (!CollectionUtils.isEmpty(reviewCounts)) {
						Map<String, Object> map = reviewCounts.get(0);
						Integer totalCount = (Integer) map.get("totalCount");
						Integer fiveStarCount = (Integer) map.get("fiveStarCount");
						int goodRate = (int) (((float) fiveStarCount / (float) totalCount) * 100);
						product.setGoodRate(goodRate);
						product.setCommentAmount(totalCount + "");


					}


				}

			} catch (IOException e) {
				e.printStackTrace();
			}


			product.setTitle(title);
			product.setUrl(url);
			product.setImgUrl(skuImg);
			product.setSource("苏宁");
			System.out.println(product);
		}






	}

	private Site site = Site.me()
			.setTimeOut(5000)
			.setCycleRetryTimes(3)
			.setCharset("UTF-8")

//			.addHeader("cookie","__jda=122270672.1618813198922953033400.1618813199.1618813199.1618813199.1; __jdv=122270672|direct|-|none|-|1618813198923; __jdc=122270672; __jdu=1618813198922953033400; shshshfp=00b5ad0f8c65d109e3bae6d77ad4d544; shshshfpa=21721a62-f4b4-c09e-6779-d336fbd4ac86-1618813199; shshshfpb=pdW4aJvYi6PDqH9kAL4RPDg%3D%3D; areaId=12; ipLoc-djd=12-904-3375-0; pinId=5OkNvRrcZOy1ffsfn98I-w; pin=mmco1997; unick=%E5%8F%AB%E6%88%91%E8%B6%85%E5%93%A5%E5%93%A5%E5%91%80%E4%B8%B6; ceshi3.com=103; _tp=0VKEU3g5B8SUkYyG4kcuaw%3D%3D; _pst=mmco1997; TrackID=11rsOLpHsQqVBCKFoRjIvpvo0eK9KbaL_3llsF9ENukHVfDMlas63MYnpqqZpgbXAc2Css45ahXXflhlNEwvMi9yBQlNjq7SYGnLpC7PXwSc; thor=A3A7E5627ACE14CF3B0D4262FD67517208145A4616B7A13789D8B9E62CBA73F3316DB596F398800567F44E1092812F7D6A1E8016CCDCED81307B25C112EBC764A1F528F590B0B0939EE0476157E9E05E1442FDA021A53D766488AC0A6265190D7953651D9EF2ABF58A84A98A235212B09A160467CD74CCBD1A7FAC2273F8ABB875D6FC7718100FF0431BF3FB43417780; wlfstk_smdl=ob36y8wmilnpbb35ple6x37rtz2z6z5v; 3AB9D23F7A4B3C9B=IXKHURP7E2GBQ2VHUSTLDOYZCHGXXVLEIB42WREEZMRGCDYDVMCEB4RVPHNTFDHNREEJQ3KUCH3RF2T244WWTOBQTQ; shshshsID=0597fd3a49198ee9e8faa0ab5d97a727_5_1618815451220; __jdb=122270672.23.1618813198922953033400|1.1618813199")
			.setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36");

	@Override
	public Site getSite() {
		return site;
	}

	public static void main(String[] args) {

		String keyWord = "小米手机";
		String minPrice = "2000";
		String maxPrice = "5000";
		List<String> urls = new ArrayList<>();

		for (int i = 1; i <= 20; i++) {

			String url = "https://search.gome.com.cn/search?question=" + keyWord + "&price=" + minPrice + "x" + maxPrice + "&priceTag=1&page=" + i;
			urls.add(url);

		}
//		https://search.gome.com.cn/search?question=%E5%B0%8F%E7%B1%B3%E6%89%8B%E6%9C%BA&instock=1&price=2000x5000&priceTag=1&page=2

		HttpClientDownloader downloader = new HttpClientDownloader();
		downloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("178.63.126.11",1080)));
		Spider.create(new GMProcessor())
//				.setDownloader(downloader)
				.addUrl((urls.toArray(new String[urls.size()])))
//				.thread(1)
				.run();


	}
}
